{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.5096085,  0.       ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhU0lEQVR4nO3da2xT9/0/8PfxNXYS2yE3E+I0aYEGyj1XkwAFUlKatdB201b1QdVNrWjDVNqqUiP9SzVpUqrtydb9qvbBtLYPplExja5lkDYkkBDiXJwQCAFCYdCEJk6AYDsXx7fz+T9oc4bLpbmRE9ufl3QEPudr++Pj+K1zvuec7xGIiMAYYzJQyF0AYyx6cQAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmQjWwB98MEHyMzMRExMDAoKCtDS0iJXKYwxmcgSQJ999hneeOMNvPvuu2hvb8fq1atRWlqKwcFBOcphjMlEkONi1IKCAuTl5eH//u//AACiKMJiseC3v/0t3n777bkuhzEmE9Vcv6HP50NbWxsqKiqkeQqFAiUlJbDZbHd8jtfrhdfrlR6LooihoSEkJiZCEIT7XjNjbGqICMPDw0hLS4NCcfcdrTkPoOvXryMYDCI1NTVkfmpqKs6fP3/H51RWVuJ3v/vdXJTHGJtFvb29SE9Pv+vyOQ+g6aioqMAbb7whPXa5XMjIyEBvby8MBoOMlTHG7sTtdsNisSA+Pv6e7eY8gJKSkqBUKjEwMBAyf2BgAGaz+Y7P0Wq10Gq1t803GAwcQIzNYz/VRTLnR8E0Gg1ycnJQU1MjzRNFETU1NbBarXNdDmNMRrLsgr3xxht44YUXkJubi/z8fPzpT3/C6OgoXnzxRTnKYYzJRJYA+uUvf4lr165h7969cDgcWLNmDaqqqm7rmGaMRTZZzgOaKbfbDaPRCJfLxX1AjM1Dk/2NhsVRMMbY/efzfYdg8CbUaguUSj0A1X0/z44DiDEGAHA6P0df3/+DSpWCmJhs6PWrodOtQkzMI1CrU6FUGjFx3Gq2gokDiDH2AxHBoBPBoBNe7wW4XF8AUEGhiIFGY4FWuxixsfkwGEoRG5s3K+/IAcQYuwsFVCoTVKqF0OkekbaI1Oq0WXsHDiDGGADA71chJmYNdLplP4TNWmi1WVCr06BQxABQzHqfEAcQYwwA0N6ejPXr/40FCxZBEJRz8p48IiJjDADg9ysgCHFzFj4ABxBjTEYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2fC1YIyxn3TrwKlEFHJR6kwuUOUAYoyFICK43W6MjIygv78f165dw/j4OFwuF3Q6HUZGRqDT6aBWqxETE4MFCxYgLi4OixYtQnx8PDQazaTfiwOIsShHRPD7/RgaGsJXX30FURTh8XhgNpuRnJyMxYsXIyEhAWq1OuQ2y6IoYnx8HENDQ+jv78fly5fhcrkQGxuLhISESb03BxBjUYiIIIoi+vv7cfLkSVy/fh0xMTHIyclBRkZGyI1A77WLZTQakZqaiuzsbGme0+lEd3f3pOrgAGIsyni9Xly4cAF2ux1msxnr1q1DSkoKVKrpD0J/6/MSEhKwfPnyST2PA4ixKDCxm9XZ2YkzZ84gOzsbv/jFLxAXFydrXRxAjEW4YDCIrq4udHZ2YtmyZXjuueegVqvv+y13JoMDiLEIRUT47rvv0NraiqSkJOzcuRN6vX5eBM8EDiDGIpDP50NLSwsGBgawZcsWmEymeRU8EziAGIsgRIS+vj4cPXoUubm5sFqtUCrnboznqeIAYixCBINBdHR0YGBgAE899RTi4+Pn5VbPraZ8LVh9fT2efPJJpKWlQRAEfP755yHLiQh79+7FwoULodPpUFJSgm+++SakzdDQEJ5//nkYDAaYTCb85je/wcjIyIw+CGPRbHR0FF999RUEQUBpaSkMBsO8Dx9gGgE0OjqK1atX44MPPrjj8j/84Q94//338dFHH6G5uRmxsbEoLS3F+Pi41Ob5559HV1cXqqurcfDgQdTX1+Pll1+e/qdgLEoREXp7e3HkyBGsXbsWa9eunde7XLehGQBABw4ckB6Lokhms5n++Mc/SvOcTidptVr6xz/+QUREZ8+eJQDU2toqtTl8+DAJgkDffffdpN7X5XIRAHK5XDMpn7GwJooinT9/nv75z3/SyMgIiaIod0mSyf5GZ3U4jsuXL8PhcKCkpESaZzQaUVBQAJvNBgCw2WwwmUzIzc2V2pSUlEChUKC5ufmOr+v1euF2u0MmxqJZIBCAzWbDtWvX8NRTTyE2NjYsdrl+bFYDyOFwAABSU1ND5qempkrLHA4HUlJSQparVCosWLBAavNjlZWVMBqN0mSxWGazbMbCit/vx/Hjx6FWq1FUVAS1Wi13SdMWFgOSVVRUwOVySVNvb6/cJTE254gI4+PjqKurQ0ZGBnJzc8Nyq+dWsxpAZrMZADAwMBAyf2BgQFpmNpsxODgYsjwQCGBoaEhq82NarRYGgyFkYiyaEBG8Xi+qq6vx8MMP48EHHwz78AFmOYCysrJgNptRU1MjzXO73WhubobVagUAWK1WOJ1OtLW1SW1qa2shiiIKCgpmsxzGIsbo6Chqa2tRWFiI9PT0iAgfYBonIo6MjODixYvS48uXL6OjowMLFixARkYG9uzZg9///vdYsmQJsrKy8M477yAtLQ07d+4EACxbtgyPP/44XnrpJXz00Ufw+/3YvXs3fvWrXyEtLW3WPhhjkYCIMDIygpqaGmzcuBELFiyQu6TZNdXDa0ePHiUAt00vvPACEX1/aPCdd96h1NRU0mq1tHXrVuru7g55jRs3btBzzz1HcXFxZDAY6MUXX6Th4eFJ18CH4Vm0cLvddPjwYbp58+a8Osz+Uyb7GxWIbhltOky43W4YjUa4XC7uD2IRiYjgdDpx4sQJFBcXw2QyyV3SlEz2NxoWR8EYiyZEhLGxMZw4cQJWqxVGo1Huku4bvhiVsXlmfHwcx44dw/r16yOvz+dHeAuIsXnE6/WipqYGubm5k76zRDjjAGJsnpg4w3nNmjVISUmJmEPt98IBxNg84Pf7UVNTgwcffBCLFi2KivABOIAYkx0RoaWlBWazGVlZWVETPgAHEGOyIiJ0dnZCo9Fg9erVURU+AAcQY7IhIpw5cwZ9fX3IycmJuvABOIAYk01/fz/6+vqwZcuWkHuuR5Po/NSMyWziguyNGzdCo9HIXY5sOIAYm2MejweHDh2C1WpFTEyM3OXIKqzPhA7Dy9hYlAsGg6ivr8emTZuQlJQkdzmyC+stILfbzSHEwoYoirDZbEhPT+ehZ34Q1gHU2NgIn88ndxmM/SQiwrlz5+Dz+bB8+fKoPOJ1J2EdQNnZ2aipqUEgEJC7FMbuiogwMDCA3t5ebNiwgcPnFmEdQJmZmUhKSkJHRwfvirF5a2RkBDabDRs3bgzrO1jcD2EdQIIgICcnB263G99++y2HEJt3/H4/jh07hry8POh0OrnLmXfCOoAAQKlUori4GHa7HTdv3pS7HMYkoiiiqakJS5cujaoLTKci7AMIANRqNR599FEcP34cXq9X7nIYky6zUCgUWLJkCYfPXUREAAmCgMTERKxcuRKNjY0QRVHuklgUIyJcu3YNly9fRkFBQdReZjEZEbNmBEFAVlYWYmNjcfLkSe4PYrLxeDxoamrCli1boFKF9bm+913EBBDwfQitW7cODocDDoeDQ4jNuUAggNraWqxbtw7x8fFylzPvRVQAAYBKpcLmzZvR3NwMj8cjdzksihAR7HY7kpOTsWjRIrnLCQsRF0AAoNPpkJeXhyNHjsDv98tdDosCRISrV69ieHgYubm53Ok8SREZQIIgIC0tDWlpadwfxOaE2+1GS0sLNmzYAKVSKXc5YSMiAwj4PoTWrl2L4eFh9PT0yF0Oi2CBQAA1NTUoKCiAVquVu5ywErEBBHx/kmJRURHsdjuGhoZ4S4jNuol+n6VLlyI9PZ13vaYoogMIALRaLaxWK+rq6hAMBuUuh0WYS5cuYWxsDMuXL5e7lLA0pQCqrKxEXl4e4uPjkZKSgp07d6K7uzukzfj4OMrLy5GYmIi4uDg8++yzGBgYCGnT09ODsrIy6PV6pKSk4K233rpvV7QLgoCFCxdiyZIlsNvtvBXEZo3T6URLSwsKCwv5ZMNpmtJaq6urQ3l5OZqamlBdXQ2/349t27ZhdHRUavP666/jyy+/xP79+1FXV4e+vj4888wz0vJgMIiysjL4fD40Njbi008/xSeffIK9e/fO3qf6EUEQ8Mgjj2BkZASXLl3iEGIz5vf70dDQgMceewx6vV7ucsIXzcDg4CABoLq6OiIicjqdpFaraf/+/VKbc+fOEQCy2WxERHTo0CFSKBTkcDikNh9++CEZDAbyer2Tel+Xy0UAyOVyTanesbEx+vvf/05ut3tKz2PsVqIo0okTJ6irq4tEUZS7nHlpsr/RGW03ulwuAMCCBQsAAG1tbfD7/SgpKZHaZGdnIyMjAzabDQBgs9mwcuVKpKamSm1KS0vhdrvR1dV1x/fxer1wu90h03TodDqUlJSgvr6eBzFj00JE+Oabb3Djxg1kZ2dzp/MMTTuARFHEnj17UFRUhBUrVgAAHA4HNBoNTCZTSNvU1FQ4HA6pza3hM7F8YtmdVFZWwmg0SpPFYplu2UhOTobFYuH+IDYtw8PDOHv2LB577DHu95kF016D5eXlOHPmDPbt2zeb9dxRRUUFXC6XNPX29k77tQRBwIoVK3Dt2jVcuXKFQ4hNWiAQQH19PYqKiqL+djqzZVoBtHv3bhw8eBBHjx5Fenq6NN9sNsPn88HpdIa0HxgYgNlsltr8+KjYxOOJNj+m1WphMBhCpplQKBQoKSnB6dOnMTIyMqPXYtGBiFBfX4+0tDS+nc4smlIAERF2796NAwcOoLa2FllZWSHLc3JyoFarUVNTI83r7u5GT08PrFYrAMBqtaKzsxODg4NSm+rqahgMhjk9l0Kn06GgoAANDQ3cH8TuiYjQ09ODQCCANWvWcL/PbJpKz/Yrr7xCRqORjh07Rv39/dI0NjYmtdm1axdlZGRQbW0t2e12slqtZLVapeWBQIBWrFhB27Zto46ODqqqqqLk5GSqqKiY9R72nyKKIrW1tdGJEyf4aAa7q+HhYfrXv/5FHo9H7lLCxmR/o1MKIAB3nD7++GOpjcfjoVdffZUSEhJIr9fT008/Tf39/SGvc+XKFdq+fTvpdDpKSkqiN998k/x+/6x/uMkIBAL09ddfU09Pz4xfi0Uen89H+/fvp76+PrlLCSuT/Y0KROHXC+t2u2E0GuFyuWbcHwR8P4JdVVUVtm3bBr1ez5vYDMD3u15tbW3SQHf8dzF5k/2N8nFEADExMcjPz8eRI0d4PGkmuXr1Km7evMn9PvcRBxD+N35Qeno6Tp06xYfmGdxuN5qbm1FUVMTj+9xHHEA/EAQBa9aswcDAAPr7++Uuh8koGAyioaEBhYWFfDPB+4wD6BZKpRIbN25EQ0NDyAW2LHoQETo6OrBw4UK+meAc4AD6Eb1eL50fxP1B0YWI0Nvbi2vXrmHVqlUcPnOAA+hHBEFARkYGEhMT0dnZyf1BUcTj8aC5uZnHdZ5DHEB3MDGe9NWrV/Hdd99xCEWBYDCI+vp6rF+/HrGxsXKXEzU4gO5CqVSiuLgYLS0tfH+xCEdEOHXqFFJSUpCWliZ3OVGFA+gejEYj8vLy0NDQwONJRygiQl9fH65evYrVq1dzv88c4wD6Cenp6UhMTER7ezvvikUgj8eD1tZWbN26lcf3kQGv8Z8gCAJWr14Nh8OBvr4+DqEIEggEcPz4ceTn5yM2Npa3fmTAATQJKpUKW7du5f6gCEI/3M8rMTERCxculLucqMUBNEkT4wcdO3aMxw+KAL29vRgZGcHatWt5y0dGHECTNHF/seTkZL7ffBgjIgwPD6O9vR3FxcV8vo/MOICmYGJYBrfbzfebD1PBYBCHDh1CYWEh38d9HuAAmqKJ+813dHTg5s2bvCUURkRRREtLC1auXAmz2cy7XvMAB9A0aLVaFBUVoa6ujvuDwgQR4eLFiwgEAsjOzpa7HPYDDqBpEAQBiYmJeOSRR2Cz2fii1XmOiDA4OAi73c73cZ9n+JuYJkEQsHjxYiiVSnR1dfGu2Dzm9XrR1NSEn/3sZ9BoNHKXw27BATQDgiAgPz8fFy5c4EHM5qlgMIjjx49j3bp1iI+Pl7sc9iMcQDOkVqvx+OOPo7W1lW9yOM9MDCpvMpmQnp7Onc7zEAfQLNDr9bBarairq4Pf75e7HIbvw+e///0vPB4P39FiHuMAmgWCICA5ORmZmZloamriTmmZERFu3LiBs2fPYv369Xyy4TzGATRLBEHA8uXLodVquVNaZh6PB1999RWKi4uhUqnkLofdAwfQLBIEAbm5uejt7cWVK1c4hGQQCARQX1+PLVu2ICEhgXe95jkOoFmmUCiwadMmnDx5ks+UnmOiKKKxsRGZmZkwm81yl8MmgQPoPoiNjUVJSQkaGhowPj4udzlRgYjQ2dkJpVKJhx9+mLd8wsSUAujDDz/EqlWrYDAYYDAYYLVacfjwYWn5+Pg4ysvLkZiYiLi4ODz77LMYGBgIeY2enh6UlZVBr9cjJSUFb731VkRezhAfH4+cnBxUVVXB5/PJXU5EIyJcuXIFQ0NDKCws5PAJI1MKoPT0dLz33ntoa2uD3W7Hli1bsGPHDnR1dQEAXn/9dXz55ZfYv38/6urq0NfXh2eeeUZ6fjAYRFlZGXw+HxobG/Hpp5/ik08+wd69e2f3U80DE7d7XrZsGRobG/nI2H1CRHA4HOjq6uLbKIcjmqGEhAT661//Sk6nk9RqNe3fv19adu7cOQJANpuNiIgOHTpECoWCHA6H1ObDDz8kg8FAXq930u/pcrkIALlcrpmWf9+Jokjt7e1kt9tJFEW5y4k4breb/v3vf9PIyIjcpbBbTPY3Ou0+oGAwiH379mF0dBRWqxVtbW3w+/0oKSmR2mRnZyMjIwM2mw0AYLPZsHLlSqSmpkptSktL4Xa7pa2oSDMxpvTQ0BBOnz7NndKzaGxsDMePH8eGDRv4Xl5hasoB1NnZibi4OGi1WuzatQsHDhzA8uXL4XA4oNFoYDKZQtqnpqbC4XAAABwOR0j4TCyfWHY3Xq8Xbrc7ZAonCoUCmzdvxtDQEB+enyVerxdVVVVYvXr1bX9zLHxMOYAefvhhdHR0oLm5Ga+88gpeeOEFnD179n7UJqmsrITRaJQmi8VyX9/vflCpVCguLsa5c+fgcDg4hGYgGAyioaEB69atw6JFi7jTOYxNOYA0Gg0WL16MnJwcVFZWYvXq1fjzn/8Ms9kMn88Hp9MZ0n5gYEA6J8NsNt92VGzi8b3O26ioqIDL5ZKm3t7eqZY9L6jVajz66KOw2+24ceOG3OWEpYnwSU9PxwMPPCB3OWyGZnwekCiK8Hq9yMnJgVqtRk1NjbSsu7sbPT09sFqtAACr1YrOzk4MDg5Kbaqrq2EwGLB8+fK7vodWq5UO/U9M4Uqv12Pz5s1obW3FzZs35S4nrIiiCLvdDp1Oh6VLl/KWTySYSs/222+/TXV1dXT58mU6ffo0vf322yQIAn399ddERLRr1y7KyMig2tpastvtZLVayWq1Ss8PBAK0YsUK2rZtG3V0dFBVVRUlJydTRUXFfelhn69EUSSn00n79u2joaEhPjo2CcFgkOx2O3V0dFAwGJS7HPYTJvsbnVIA/frXv6YHHniANBoNJScn09atW6XwISLyeDz06quvUkJCAun1enr66aepv78/5DWuXLlC27dvJ51OR0lJSfTmm2+S3++fShlhH0ATbty4QVVVVeR2u+UuZV4LBoPU2NhIJ06c4PAJE5P9jQpE4dcb6na7YTQa4XK5wnp3DABu3ryJ48ePo7i4mC+evANRFNHZ2Qmfz4ecnBwezzlMTPY3yt+mzEwmE6xWKxobGzE8PCx3OfOKKIpoamqC3+/n8IlQ/I3KbGIws6KiIjQ2NmJoaIgP0eP78GlvbwcArFu3jsMnQvG3Ok+YTCYUFBTgyJEjUX+IPhgMoqmpCWq1mm+jE+H4m50nBEFAQkICtm/fjtbWVvT390fllpDf70ddXR20Wi1WrVrF4RPh+NudZ+Li4rB582a0t7fj4sWLURVCo6OjqK2tRWZmJg8kHyU4gOYZQRAQExODbdu2ob+/H6dPn474oTyICC6XC4cPH8ayZcuQlZXF4RMl+DD8PBYMBtHS0gKPx4MNGzZArVbP6uvf+tVP/P/WecFgED6fDz6fD16vN+TfQCCAZcuWQavVzriGb7/9FqdPn0ZxcTEWLFgwo9dj88Nkf6N8y4B5TKlUorCwEJcuXUJ1dTWKi4sRHx8/K1sHwWAQn332GXp6ejA8PIyRkRGMjIxI/x8eHobH47kteCb+VavVaG5uxpIlS6ZdgyiK6OrqwoULF1BWVjbjMGPhhwNonhMEAQ899BAMBgOqqqpQWFgIi8Uy4xAKBoOorKzEmTNnpvV8vV6P0dHRaT2XiOD1etHQ0ACDwYAdO3bw7XOiFPcBhYGJc4WeeuopdHd3o6GhQfZxtEVRnNatqIkIg4ODOHjwINLS0pCXl8fhE8U4gMLEROf01q1bERMTgy+++ALXr1+X7SiZKIpTPnPb7/fj+PHjaG5uRmlpKZYtW8adzVGOAyjMKBQK5OXlYfPmzairq0NDQwP8fv+Ug0gQBCxatEh6rNfrkZycfNcOw/j4eCQnJ0Ov1wMAfD4fzp8/P6n3IiL09fXhP//5D/R6PcrKymatL4uFN972DVMmkwk7duzA6dOn8cUXXyA3NxcWi2XSJ+4pFApkZGQAAFasWIFNmzbBZDLB4/Ggra0NDQ0NCAaDUCqVKCoqQk5ODvR6PZxOJ+rq6nDmzJlJ3W5obGwMra2t6Ovrw/bt22E0Gjl4mIQDKEwJggCVSoW1a9di8eLFaGhoQGdnJwoLC5GYmPiTP3JBEBAfHw+LxYInnnhC2rJRq9XYsGEDxsbGYLfbsXbtWmzcuFHqp0lOTsYTTzxxz3G5iQiBQACnTp3ChQsXkJubi+LiYr5lDrsNB1CYEwQBBoMB27dvx+DgIKqrqxEfH4+CggIkJSXdNYgEQUBcXBxyc3Oh0+lClqlUKuTl5eHs2bPIz8+/rZNYp9OhuLj4jrtrXq8XnZ2duHjxIh566CE888wz0Gq1vNXD7ohPRIwwwWAQ/f39aGlpQSAQQH5+PhYtWgSVShUSAkSE/fv3w2az3fGuEmq1GllZWbhy5codd7Ueeugh/PznP0dMTAyICMPDw+js7ERvby/S09Oxdu1a6PV6Dp4oNdnfKAdQBKLvR7rEtWvXpK2RZcuWYenSpUhJSYFCoYAgCCAinDp1Cl9++eVtl3vk5eVh27ZtqK6uRktLS8gyhUKBJ598EkuXLsXFixdx9uxZKBQKLFmyBI888gjUajUHT5TjAGIA/nfSX09PDzo6OuDxeJCUlIS0tDQ88MAD0Gg0aGhoQHt7O/x+P5RKJR588EHs2LEDsbGxGBsbw+eff45Lly4hGAxCFEXodDokJydDp9PBYrEgOzsbSUlJfOU6k/ClGAzA/84fWrp0KZYsWQKfz4fBwUFcvHgRly9fhsPhQExMjHTdV0pKCrRaLWw2G5xOJxQKBcbGxqDT6ZCZmYmEhASkp6cjOTkZJpOJQ4fNCAdQFBEEAVqtFhaLBRaLBUQEURSlkwo9Hg/8fj+8Xi80Gg2ICAaDAXFxcdJu1cTE2GzgAIpigiBAqVRCqVTyVehMFrz9zBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2YXkpxsQF/PcalY8xJp+J3+ZPDbYRlgF048YNAIDFYpG5EsbYvQwPD8NoNN51eVgG0MSFkz09Pff8cCyU2+2GxWJBb28vj6M0SbzOpmdilMy0tLR7tgvLAJoYg8ZoNPIfxTQYDAZeb1PE62zqJrNxwJ3QjDHZcAAxxmQTlgGk1Wrx7rvvQqvVyl1KWOH1NnW8zu6vsByUnjEWGcJyC4gxFhk4gBhjsuEAYozJhgOIMSabsAygDz74AJmZmYiJiUFBQcFttw6OJpWVlcjLy0N8fDxSUlKwc+dOdHd3h7QZHx9HeXk5EhMTERcXh2effRYDAwMhbXp6elBWVga9Xo+UlBS89dZbCAQCc/lRZPPee+9BEATs2bNHmsfrbI5QmNm3bx9pNBr629/+Rl1dXfTSSy+RyWSigYEBuUuTRWlpKX388cd05swZ6ujooCeeeIIyMjJoZGREarNr1y6yWCxUU1NDdrudCgsLaf369dLyQCBAK1asoJKSEjp58iQdOnSIkpKSqKKiQo6PNKdaWlooMzOTVq1aRa+99po0n9fZ3Ai7AMrPz6fy8nLpcTAYpLS0NKqsrJSxqvljcHCQAFBdXR0RETmdTlKr1bR//36pzblz5wgA2Ww2IiI6dOgQKRQKcjgcUpsPP/yQDAYDeb3euf0Ac2h4eJiWLFlC1dXVtGnTJimAeJ3NnbDaBfP5fGhra0NJSYk0T6FQoKSkBDabTcbK5g+XywXgfxfstrW1we/3h6yz7OxsZGRkSOvMZrNh5cqVSE1NldqUlpbC7Xajq6trDqufW+Xl5SgrKwtZNwCvs7kUVhejXr9+HcFgMORLB4DU1FScP39epqrmD1EUsWfPHhQVFWHFihUAAIfDAY1GA5PJFNI2NTUVDodDanOndTqxLBLt27cP7e3taG1tvW0Zr7O5E1YBxO6tvLwcZ86cQUNDg9ylzGu9vb147bXXUF1djZiYGLnLiWphtQuWlJQEpVJ529GIgYEBmM1mmaqaH3bv3o2DBw/i6NGjSE9Pl+abzWb4fD44nc6Q9reuM7PZfMd1OrEs0rS1tWFwcBDr1q2DSqWCSqVCXV0d3n//fahUKqSmpvI6myNhFUAajQY5OTmoqamR5omiiJqaGlitVhkrkw8RYffu3Thw4ABqa2uRlZUVsjwnJwdqtTpknXV3d6Onp0daZ1arFZ2dnRgcHJTaVFdXw2AwYPny5XPzQebQ1q1b0dnZiY6ODmnKzc3F888/L/2f19kckbsXfKr27dtHWq2WPvnkEzp79iy9/PLLZDKZQo5GRJNXXnmFjEYjHTt2jPr7+6VpbGxMarNr1y7KyMig2tpastvtZLVayWq1SssnDilv27aNOjo6qKqqipKTk6PqkPKtR8GIeJ3NlbALICKiv/zlL5SRkUEajYby8/OpqalJ7pJkA+CO08cffyy18Xg89Oqrr1JCQgLp9Xp6+umnqb+/P+R1rly5Qtu3byedTkdJSUn05ptvkt/vn+NPI58fBxCvs7nBw3EwxmQTVn1AjLHIwgHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2fx/TWq0T7eGbNsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.45392898  0.        ]\n",
      "action= [0.97239244]\n",
      "next_state= [-0.45298913  0.00093986]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.sac.sac.SAC at 0x7f91759ef9a0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import SAC\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = SAC(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    buffer_size=10_0000,  #reply_buffer_size\n",
    "    learning_starts=100,  #积累N步的数据以后开始训练\n",
    "    batch_size=256,  #每次采样N条数据\n",
    "    tau=5e-3,  #target网络软更新系数\n",
    "    gamma=0.9,\n",
    "    train_freq=(1, 'step'),  #训练频率\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/7.SAC.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-132.9, 108.11470760261992)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = SAC.load('save/7.SAC.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhg0lEQVR4nO3dfWwT9/0H8PfZ8VMe7DzHhCQkLYEA4TGPJhRKSJMy1pa2mtauf1Td1Io2TKWtKhFppZo0KdX2x7Z2Ff1jWts/pjKxjW1FkDYkkBDiPDhpIAQamhaIIbEDGNt5cBzb9/n90eZ+hKcmEHJ2/HlJJ9V3X9sfX7h377539z2BiAiMMSYDhdwFMMbCFwcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2sgXQhx9+iMzMTGi1WhQVFaGtrU2uUhhjMpElgP7xj3/gzTffxLvvvovOzk6sXr0aFRUVGBoakqMcxphMBDluRi0qKkJBQQH+8pe/AABEUUR6ejp+/etfY/fu3XNdDmNMJhFz/YUTExPo6OhAVVWVNE+hUKCsrAxms/m27/F6vfB6vdJrURThcDiQkJAAQRAeeM2MsZkhIgwPDyM1NRUKxZ0PtOY8gK5evYpAIICUlJQp81NSUvD111/f9j3V1dX47W9/OxflMcZmkdVqRVpa2h2Xz3kA3Yuqqiq8+eab0muXy4WMjAxYrVbo9XoZK2OM3Y7b7UZ6ejpiYmLu2m7OAygxMRFKpRJ2u33KfLvdDqPReNv3aDQaaDSaW+br9XoOIMaC2I91kcz5WTC1Wo28vDzU1dVJ80RRRF1dHUwm01yXwxiTkSyHYG+++SZefPFF5Ofno7CwEH/6058wOjqKl156SY5yGGMykSWAfv7zn+PKlSvYs2cPbDYb1qxZg5qamls6phlj85ss1wHdL7fbDYPBAJfLxX1AjAWh6W6jIXEWjDH24E1MXEYgcB0qVTqUykgAEQ/8OjsOIMYYAMDp/A8GBn6DiIhkaLU5iIxcDZ1uFbTaFVCpUqBUGjB53mq2gokDiDH2AxGBgBOBgBNe7zm4XP8DEAGFQgu1Oh0azWJERRVCr69AVFTBrHwjBxBj7A4UiIiIRUTEAuh0K6Q9IpUqdda+gQOIMQYA8PkioNWugU637IewWQuNJgsqVSoUCi0Axaz3CXEAMcYAAJ2dSVi//r+Ij18IQVDOyXdyADHGAAA+nwKCED0lfAKBAIaHhwEAsbGxs/6dHECMMRARRFHE2NgYHA4HvvnmG5w6dQodHR04d+4cTCYTPvjgA0REzG5kcAAxFkZuvO54bGwMV69eRV9fH7q6unD48GH85je/gcPhgMPhQCAQkNoODw9jdHQUBoNhVuvhAGIsDIiiCJvNhnPnzqGnpwcWiwVnzpzBhQsX4HQ64fP5cLebIlwuF+x2OwcQY2zm3G43tm7dinPnzmF8fHzG73c4HOjq6kJ2dvasngnjx/IwFga0Wi3S0tKmHT6CIECn00Gn00GhUICI0NraOut18R4QY2FAo9EgKytrWm11Oh02bdqEpUuXQhAEfPfddzh69ChOnjwJURShVM7eKXreA2IsDAiCgM2bN0sDxCuVSmRmZqK4uBi5ubnQ6XTS/MceewxFRUWIi4tDbGws1q5di61bt6K/v3/WH53Fe0CMhYklS5ZAp9PB4/GgtLQUBQUFUKlUEEURVqsV//rXv6DX67Fy5cop/TyCIGDp0qVQKpUQRXFWa+IAYixMZGVlSYdhxcXF0qGUUqnEokWLUFhYiIsXL972Wh+lUondu3cjNXX27gMD+BCMsbCh0+mwaNEiZGdn3/KsLkEQsGDBAvzxj3+87RXPUVFRMBqNs34vGAcQY2FCoVDgueeeQ2Zm5m2DRKPRYP369SgvL5f6hCbnl5aWIjExUbpimoik6X7wIRhjYUIQBPziF79AaWkpPvvsM4yMjEjLFAoFkpOTAQA5OTkQBAGdnZ24fPkyDAYDzp8/j1OnTkGn02FkZAQ6nQ4qlQparRbx8fGIjo7GwoULERMTA7VaPe2aOIAYCyMKhQILFizAE088gaNHj8LhcECr1SI3Nxd9fX344osvIIoiPB4PjEYjFi9ejPj4eMTFxUGlUk05dBNFEePj43A4HBgcHMT58+fhcrkQFRWFuLi4adXDg9IzFoaICGNjY/j222/R09MDj8cDtVqNwsJCZGRkTHkQ6HT6fW6MEafTid7eXphMJh6UnjE2ldfrxblz52CxWGA0GrFx40YkJycjIuLeB6G/8X1xcXFYvnz5tN7HAcRYGCAi+Hw+dHd34/Tp08jJycHPfvYzREdHy1oXBxBj81wgEEBPTw+6u7uxbNkyPP/881CpVA/8kTvTwQHE2DxFRLh8+TLa29uRmJiI7du3IzIyMiiCZxIHEGPz0MTEBNra2mC321FaWorY2NigCp5JHECMzSNEhIGBARw9ehT5+fkwmUyzevf6bOMAYmyeCAQC6Orqgt1ux5NPPomYmJig3Ou50YxvxWhsbMQTTzyB1NRUCIKA//znP1OWExH27NmDBQsWQKfToaysDN98882UNg6HAy+88AL0ej1iY2Pxq1/9aspVmYyxmRkdHcUXX3wBQRBQUVEBvV4f9OED3EMAjY6OYvXq1fjwww9vu/z3v/893n//fXz00UdobW1FVFQUKioqpozE9sILL6Cnpwe1tbU4ePAgGhsb8corr9z7r2AsTBERrFYrjhw5grVr12Lt2rVBfch1C7oPAOjAgQPSa1EUyWg00h/+8AdpntPpJI1GQ5999hkREZ05c4YAUHt7u9Tm8OHDJAgCXb58eVrf63K5CAC5XK77KZ+xkCaKIn399df0z3/+k0ZGRkgURblLkkx3G53Vu+HPnz8Pm82GsrIyaZ7BYEBRURHMZjMAwGw2IzY2Fvn5+VKbsrIyKBSKO4456/V64Xa7p0yMhTO/3w+z2YwrV67gySefRFRUVEgcct1sVgPIZrMBAFJSUqbMT0lJkZbZbDbprttJERERiI+Pl9rcrLq6GgaDQZrS09Nns2zGQorP58Px48ehUqlQUlIClUold0n3LCTGA6qqqoLL5ZImq9Uqd0mMzTkiwvj4OBoaGpCRkYH8/PyQ3Ou50awGkNFoBADY7fYp8+12u7TMaDTeMrC13++Hw+GQ2txMo9FAr9dPmRgLJ0QEr9eL2tpaLF26FA899FDIhw8wywGUlZUFo9GIuro6aZ7b7UZraytMJhMAwGQywel0oqOjQ2pTX18PURRRVFQ0m+UwNm+Mjo6ivr4excXFSEtLmxfhA9zDhYgjIyPo6+uTXp8/fx5dXV2Ij49HRkYGdu3ahd/97nfIzs5GVlYW3nnnHaSmpmL79u0AgGXLluHxxx/Hyy+/jI8++gg+nw87d+7Ec889N+sDXjMW6ogIIyMjqKurw8aNGxEfHy93SbNrpqfXjh49SgBumV588UUi+v7U4DvvvEMpKSmk0Whoy5Yt1NvbO+Uzrl27Rs8//zxFR0eTXq+nl156iYaHh6ddA5+GZ+HC7XbT4cOH6fr160F1mv3HTHcb5RERGQtCRASn04kTJ05gw4YNt31SRTCb7jYaEmfBGAsn9MNwqSdOnIDJZILBYJC7pAeGb0ZlLMiMj4/j2LFjWL9+/fzr87kJ7wExFkS8Xi/q6uqQn58/7SdLhDIOIMaCxOQVzmvWrEFycvK8OdV+NxxAjAUBn8+Huro6PPTQQ1i4cGFYhA/AAcSY7IgIbW1tMBqNyMrKCpvwATiAGJMVEaG7uxtqtRqrV68Oq/ABOIAYkw0R4fTp0xgYGEBeXl7YhQ/AAcSYbAYHBzEwMIDS0tIpz1wPJ+H5qxmT2eQN2Rs3boRarZa7HNlwADE2xzweDw4dOgSTyQStVit3ObIK6SuhQ/A2NhbmAoEAGhsbsWnTJiQmJspdjuxCeg/I7XZzCLGQIYoizGYz0tLSeOiZH4R0ADU3N2NiYkLuMhj7UUSEs2fPYmJiAsuXLw/LM163E9IBlJOTg7q6Ovj9frlLYeyOiAh2ux1WqxWPPPIIh88NQjqAMjMzkZiYiK6uLj4UY0FrZGQEZrMZGzduDOknWDwIIR1AgiAgLy8PbrcbFy9e5BBiQcfn8+HYsWMoKCiATqeTu5ygE9IBBABKpRIbNmyAxWLB9evX5S6HMYkoimhpacGSJUvC6gbTmQj5AAIAlUqFRx99FMePH4fX65W7HMak2ywUCgWys7M5fO5gXgSQIAhISEjAypUr0dzcDFEU5S6JhTEiwpUrV3D+/HkUFRWF7W0W0zFv1owgCMjKykJUVBS++uor7g9isvF4PGhpaUFpaSkiIkL6Wt8Hbt4EEPB9CK1btw42mw02m41DiM05v9+P+vp6rFu3DjExMXKXE/TmVQABQEREBDZv3ozW1lZ4PB65y2FhhIhgsViQlJSEhQsXyl1OSJh3AQQAOp0OBQUFOHLkCHw+n9zlsDBARLh06RKGh4eRn5/Pnc7TNC8DSBAEpKamIjU1lfuD2Jxwu91oa2vDI488AqVSKXc5IWNeBhDwfQitXbsWw8PD6O/vl7scNo/5/X7U1dWhqKgIGo1G7nJCyrwNIOD7ixRLSkpgsVjgcDh4T4jNusl+nyVLliAtLY0PvWZoXgcQAGg0GphMJjQ0NCAQCMhdDptnvv32W4yNjWH58uVylxKSZhRA1dXVKCgoQExMDJKTk7F9+3b09vZOaTM+Po7KykokJCQgOjoazz77LOx2+5Q2/f392LZtGyIjI5GcnIy33377gd3RLggCFixYgOzsbFgsFt4LYrPG6XSira0NxcXFfLHhPZrRWmtoaEBlZSVaWlpQW1sLn8+H8vJyjI6OSm3eeOMNfP7559i/fz8aGhowMDCAZ555RloeCASwbds2TExMoLm5GZ9++ik++eQT7NmzZ/Z+1U0EQcCKFSswMjKCb7/9lkOI3Tefz4empiY89thjiIyMlLuc0EX3YWhoiABQQ0MDERE5nU5SqVS0f/9+qc3Zs2cJAJnNZiIiOnToECkUCrLZbFKbvXv3kl6vJ6/XO63vdblcBIBcLteM6h0bG6O///3v5Ha7Z/Q+xm4kiiKdOHGCenp6SBRFucsJStPdRu9rv9HlcgEA4uPjAQAdHR3w+XwoKyuT2uTk5CAjIwNmsxkAYDabsXLlSqSkpEhtKioq4Ha70dPTc9vv8Xq9cLvdU6Z7odPpUFZWhsbGRh7EjN0TIsI333yDa9euIScnhzud79M9B5Aoiti1axdKSkqQm5sLALDZbFCr1YiNjZ3SNiUlBTabTWpzY/hMLp9cdjvV1dUwGAzSlJ6efq9lIykpCenp6dwfxO7J8PAwzpw5g8cee4z7fWbBPa/ByspKnD59Gvv27ZvNem6rqqoKLpdLmqxW6z1/liAIyM3NxZUrV3DhwgUOITZtfr8fjY2NKCkpCfvH6cyWewqgnTt34uDBgzh69CjS0tKk+UajERMTE3A6nVPa2+12GI1Gqc3NZ8UmX0+2uZlGo4Fer58y3Q+FQoGysjKcOnUKIyMj9/VZLDwQERobG5GamsqP05lFMwogIsLOnTtx4MAB1NfXIysra8ryvLw8qFQq1NXVSfN6e3vR398Pk8kEADCZTOju7sbQ0JDUpra2Fnq9fk6vpdDpdCgqKkJTUxP3B7G7IiL09/fD7/djzZo13O8zm2bSs/3qq6+SwWCgY8eO0eDgoDSNjY1JbXbs2EEZGRlUX19PFouFTCYTmUwmabnf76fc3FwqLy+nrq4uqqmpoaSkJKqqqpr1HvYfI4oidXR00IkTJ/hsBruj4eFh+ve//00ej0fuUkLGdLfRGQUQgNtOH3/8sdTG4/HQa6+9RnFxcRQZGUlPP/00DQ4OTvmcCxcu0NatW0mn01FiYiK99dZb5PP5Zv3HTYff76cvv/yS+vv77/uz2PwzMTFB+/fvp4GBAblLCSnT3UYFotDrhXW73TAYDHC5XPfdHwR8P4JdTU0NysvLERkZybvYDMD3h14dHR3SQHf872L6pruN8nlEAFqtFoWFhThy5AiPJ80kly5dwvXr17nf5wHiAML/jx+UlpaGkydP8ql5BrfbjdbWVpSUlPD4Pg8QB9APBEHAmjVrYLfbMTg4KHc5TEaBQABNTU0oLi7mhwk+YBxAN1Aqldi4cSOampqm3GDLwgcRoaurCwsWLOCHCc4BDqCbREZGStcHcX9QeCEiWK1WXLlyBatWreLwmQMcQDcRBAEZGRlISEhAd3c39weFEY/Hg9bWVh7XeQ5xAN3G5HjSly5dwuXLlzmEwkAgEEBjYyPWr1+PqKgoucsJGxxAd6BUKrFhwwa0tbXx88XmOSLCyZMnkZycjNTUVLnLCSscQHdhMBhQUFCApqYmHk96niIiDAwM4NKlS1i9ejX3+8wxDqAfkZaWhoSEBHR2dvKh2Dzk8XjQ3t6OLVu28Pg+MuA1/iMEQcDq1aths9kwMDDAITSP+P1+HD9+HIWFhYiKiuK9HxlwAE1DREQEtmzZwv1B8wj98DyvhIQELFiwQO5ywhYH0DRNjh907NgxHj9oHrBarRgZGcHatWt5z0dGHEDTNPl8saSkJH7efAgjIgwPD6OzsxMbNmzg631kxgE0A5PDMrjdbn7efIgKBAI4dOgQiouL+TnuQYADaIYmnzff1dWF69ev855QCBFFEW1tbVi5ciWMRiMfegUBDqB7oNFoUFJSgoaGBu4PChFEhL6+Pvj9fuTk5MhdDvsBB9A9EAQBCQkJWLFiBcxmM9+0GuSICENDQ7BYLPwc9yDDf4l7JAgCFi9eDKVSiZ6eHj4UC2JerxctLS346U9/CrVaLXc57AYcQPdBEAQUFhbi3LlzPIhZkAoEAjh+/DjWrVuHmJgYucthN+EAuk8qlQqPP/442tvb+SGHQWZyUPnY2FikpaVxp3MQ4gCaBZGRkTCZTGhoaIDP55O7HIbvw+e7776Dx+PhJ1oEMQ6gWSAIApKSkpCZmYmWlhbulJYZEeHatWs4c+YM1q9fzxcbBjEOoFkiCAKWL18OjUbDndIy83g8+OKLL7BhwwZERETIXQ67Cw6gWSQIAvLz82G1WnHhwgUOIRn4/X40NjaitLQUcXFxfOgV5DiAZplCocCmTZvw1Vdf8ZXSc0wURTQ3NyMzMxNGo1Huctg0cAA9AFFRUSgrK0NTUxPGx8flLicsEBG6u7uhVCqxdOlS3vMJETMKoL1792LVqlXQ6/XQ6/UwmUw4fPiwtHx8fByVlZVISEhAdHQ0nn32Wdjt9imf0d/fj23btiEyMhLJycl4++235+XtDDExMcjLy0NNTQ0mJibkLmdeIyJcuHABDocDxcXFHD4hZEYBlJaWhvfeew8dHR2wWCwoLS3FU089hZ6eHgDAG2+8gc8//xz79+9HQ0MDBgYG8Mwzz0jvDwQC2LZtGyYmJtDc3IxPP/0Un3zyCfbs2TO7vyoITD7uedmyZWhubuYzYw8IEcFms6Gnp4cfoxyK6D7FxcXRX//6V3I6naRSqWj//v3SsrNnzxIAMpvNRER06NAhUigUZLPZpDZ79+4lvV5PXq932t/pcrkIALlcrvst/4ETRZE6OzvJYrGQKIpylzPvuN1u+u9//0sjIyNyl8JuMN1t9J77gAKBAPbt24fR0VGYTCZ0dHTA5/OhrKxMapOTk4OMjAyYzWYAgNlsxsqVK5GSkiK1qaiogNvtlvai5pvJMaUdDgdOnTrFndKzaGxsDMePH8cjjzzCz/IKUTMOoO7ubkRHR0Oj0WDHjh04cOAAli9fDpvNBrVajdjY2CntU1JSYLPZAAA2m21K+Ewun1x2J16vF263e8oUShQKBTZv3gyHw8Gn52eJ1+tFTU0NVq9efcu/ORY6ZhxAS5cuRVdXF1pbW/Hqq6/ixRdfxJkzZx5EbZLq6moYDAZpSk9Pf6Df9yBERERgw4YNOHv2LGw2G4fQfQgEAmhqasK6deuwcOFC7nQOYTMOILVajcWLFyMvLw/V1dVYvXo1/vznP8NoNGJiYgJOp3NKe7vdLl2TYTQabzkrNvn6btdtVFVVweVySZPVap1p2UFBpVLh0UcfhcViwbVr1+QuJyRNhk9aWhoWLVokdznsPt33dUCiKMLr9SIvLw8qlQp1dXXSst7eXvT398NkMgEATCYTuru7MTQ0JLWpra2FXq/H8uXL7/gdGo1GOvU/OYWqyMhIbN68Ge3t7bh+/brc5YQUURRhsVig0+mwZMkS3vOZD2bSs717925qaGig8+fP06lTp2j37t0kCAJ9+eWXRES0Y8cOysjIoPr6erJYLGQymchkMknv9/v9lJubS+Xl5dTV1UU1NTWUlJREVVVVD6SHPViJokhOp5P27dtHDoeDz45NQyAQIIvFQl1dXRQIBOQuh/2I6W6jMwqgX/7yl7Ro0SJSq9WUlJREW7ZskcKHiMjj8dBrr71GcXFxFBkZSU8//TQNDg5O+YwLFy7Q1q1bSafTUWJiIr311lvk8/lmUkbIB9Cka9euUU1NDbndbrlLCWqBQICam5vpxIkTHD4hYrrbqEAUer2hbrcbBoMBLpcrpA/HAOD69es4fvw4NmzYwDdP3oYoiuju7sbExATy8vJ4POcQMd1tlP+aMouNjYXJZEJzczOGh4flLieoiKKIlpYW+Hw+Dp95iv+iMpsczKykpATNzc1wOBx8ih7fh09nZycAYN26dRw+8xT/VYNEbGwsioqKcOTIkbA/RR8IBNDS0gKVSsWP0Znn+C8bJARBQFxcHLZu3Yr29nYMDg6G5Z6Qz+dDQ0MDNBoNVq1axeEzz/FfN8hER0dj8+bN6OzsRF9fX1iF0OjoKOrr65GZmckDyYcJDqAgIwgCtFotysvLMTg4iFOnTs37oTyICC6XC4cPH8ayZcuQlZXF4RMmOICClEqlQklJCcbGxnDs2LF5+7gfIsLFixfR0NCA0tJSZGRkcPiEEQ6gIKZUKlFcXIyMjAzU1tbC7XbPq0MyURRx+vRpdHR0oLy8HHFxcXKXxOYYB1CQEwQBDz/8MPLz81FTUwOr1RryIUREGB8fR319PTweD5566ilotVre8wlDHEAhYPJaoSeffBK9vb1oamoK2XG0iQhDQ0M4ePAgUlNTUVBQwM/uCmMcQCFisnN6y5Yt0Gq1+N///oerV6+G1N6Qz+fD8ePH0draioqKCixbtoz3esIc/68nxCgUChQUFGDx4sWor69HcnIyiouLEREREbQbMxFhcHAQbW1tSEtL48HjmYRvRg1RRIRAIIBTp07h/PnzyM/PR3p6etBduDc2Nob29nYMDAxg69atMBgMQRuUbPZMdxvlAApxRITh4WE0NTVBFEUUFxcjISFB1o2ciOD3+3Hy5EmcO3cO+fn5ePjhh3mvJ4xwAIWZyc7d+vp6xMTEoKioCImJiXMeRF6vF93d3ejr68PDDz+MlStXQqPR8F5PmOEAClOBQEDqb/H7/SgsLMTChQsfaB/R5F5Yd3c3rFYr0tLSsHbtWkRGRnLwhCkOoDBG3490iStXrkh7I8uWLcOSJUuQnJwMhUJxX8Ew+U/G4/Ggr68PZ86cgUKhQHZ2NlasWAGVSsXBE+Y4gBiA78PC6/Wiv78fXV1d8Hg8SExMRGpqKhYtWgSdTicdIt0pNCb7dLxeLxwOBy5evIi+vj6o1Wqkp6cjJycHiYmJQdcBzuTDAcRuQUSYmJjA0NAQ+vr6cP36ddhsNkRHR0MQBMTExECr1cLv90Or1cLpdEKhUGB0dBSiKEKr1SIqKgrZ2dlISkpCbGwshw67reluo3wdUBgRBAEajQbp6elIT08HEUEURYiiiOHhYXg8Hvh8Pni9XqjVahAR9Ho9oqOjpcOqu+0pMTZTHEBhTBAEKJVKKJVKxMfHy10OC0O8/8wYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNiF5K8bk/bNut1vmShhjtzO5bf7Yve4hGUDXrl0DAKSnp8tcCWPsboaHh2EwGO64PCQDaPLGyf7+/rv+ODaV2+1Geno6rFYrD2MyTbzO7s3kKJmpqal3bReSATQ5Bo3BYOB/FPdAr9fzepshXmczN52dA+6EZozJhgOIMSabkAwgjUaDd999FxqNRu5SQgqvt5njdfZgheSY0Iyx+SEk94AYY/MDBxBjTDYcQIwx2XAAMcZkE5IB9OGHHyIzMxNarRZFRUVoa2uTuyTZVFdXo6CgADExMUhOTsb27dvR29s7pc34+DgqKyuRkJCA6OhoPPvss7Db7VPa9Pf3Y9u2bYiMjERycjLefvtt+P3+ufwpsnnvvfcgCAJ27dolzeN1NkcoxOzbt4/UajX97W9/o56eHnr55ZcpNjaW7Ha73KXJoqKigj7++GM6ffo0dXV10U9+8hPKyMigkZERqc2OHTsoPT2d6urqyGKxUHFxMa1fv15a7vf7KTc3l8rKyuirr76iQ4cOUWJiIlVVVcnxk+ZUW1sbZWZm0qpVq+j111+X5vM6mxshF0CFhYVUWVkpvQ4EApSamkrV1dUyVhU8hoaGCAA1NDQQEZHT6SSVSkX79++X2pw9e5YAkNlsJiKiQ4cOkUKhIJvNJrXZu3cv6fV68nq9c/sD5tDw8DBlZ2dTbW0tbdq0SQogXmdzJ6QOwSYmJtDR0YGysjJpnkKhQFlZGcxms4yVBQ+XywXg/2/Y7ejogM/nm7LOcnJykJGRIa0zs9mMlStXIiUlRWpTUVEBt9uNnp6eOax+blVWVmLbtm1T1g3A62wuhdTNqFevXkUgEJjyRweAlJQUfP311zJVFTxEUcSuXbtQUlKC3NxcAIDNZoNarUZsbOyUtikpKbDZbFKb263TyWXz0b59+9DZ2Yn29vZblvE6mzshFUDs7iorK3H69Gk0NTXJXUpQs1qteP3111FbWwutVit3OWEtpA7BEhMToVQqbzkbYbfbYTQaZaoqOOzcuRMHDx7E0aNHkZaWJs03Go2YmJiA0+mc0v7GdWY0Gm+7TieXzTcdHR0YGhrCunXrEBERgYiICDQ0NOD9999HREQEUlJSeJ3NkZAKILVajby8PNTV1UnzRFFEXV0dTCaTjJXJh4iwc+dOHDhwAPX19cjKypqyPC8vDyqVaso66+3tRX9/v7TOTCYTuru7MTQ0JLWpra2FXq/H8uXL5+aHzKEtW7agu7sbXV1d0pSfn48XXnhB+m9eZ3NE7l7wmdq3bx9pNBr65JNP6MyZM/TKK69QbGzslLMR4eTVV18lg8FAx44do8HBQWkaGxuT2uzYsYMyMjKovr6eLBYLmUwmMplM0vLJU8rl5eXU1dVFNTU1lJSUFFanlG88C0bE62yuhFwAERF98MEHlJGRQWq1mgoLC6mlpUXukmQD4LbTxx9/LLXxeDz02muvUVxcHEVGRtLTTz9Ng4ODUz7nwoULtHXrVtLpdJSYmEhvvfUW+Xy+Of418rk5gHidzQ0ejoMxJpuQ6gNijM0vHECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTzf8Bkrf9c/II2ZQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-44.0 145 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
